{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# An overview of the SMOTE(NC) Oversampling Method\n",
    "un-used by itself"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/Applications/anaconda/envs/st-ds/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Applications/anaconda/envs/st-ds/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Applications/anaconda/envs/st-ds/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Applications/anaconda/envs/st-ds/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Applications/anaconda/envs/st-ds/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Applications/anaconda/envs/st-ds/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/Applications/anaconda/envs/st-ds/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Applications/anaconda/envs/st-ds/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Applications/anaconda/envs/st-ds/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Applications/anaconda/envs/st-ds/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Applications/anaconda/envs/st-ds/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Applications/anaconda/envs/st-ds/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from imblearn.over_sampling import SMOTENC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on class SMOTENC in module imblearn.over_sampling._smote:\n",
      "\n",
      "class SMOTENC(SMOTE)\n",
      " |  SMOTENC(categorical_features, sampling_strategy='auto', random_state=None, k_neighbors=5, n_jobs=1)\n",
      " |  \n",
      " |  Synthetic Minority Over-sampling Technique for Nominal and Continuous\n",
      " |  (SMOTE-NC).\n",
      " |  \n",
      " |  Unlike :class:`SMOTE`, SMOTE-NC for dataset containing continuous and\n",
      " |  categorical features.\n",
      " |  \n",
      " |  Read more in the :ref:`User Guide <smote_adasyn>`.\n",
      " |  \n",
      " |  Parameters\n",
      " |  ----------\n",
      " |  categorical_features : ndarray, shape (n_cat_features,) or (n_features,)\n",
      " |      Specified which features are categorical. Can either be:\n",
      " |  \n",
      " |      - array of indices specifying the categorical features;\n",
      " |      - mask array of shape (n_features, ) and ``bool`` dtype for which\n",
      " |        ``True`` indicates the categorical features.\n",
      " |  \n",
      " |  sampling_strategy : float, str, dict or callable, (default='auto')\n",
      " |      Sampling information to resample the data set.\n",
      " |  \n",
      " |      - When ``float``, it corresponds to the desired ratio of the number of\n",
      " |        samples in the minority class over the number of samples in the\n",
      " |        majority class after resampling. Therefore, the ratio is expressed as\n",
      " |        :math:`\\alpha_{os} = N_{rm} / N_{M}` where :math:`N_{rm}` is the\n",
      " |        number of samples in the minority class after resampling and\n",
      " |        :math:`N_{M}` is the number of samples in the majority class.\n",
      " |  \n",
      " |          .. warning::\n",
      " |             ``float`` is only available for **binary** classification. An\n",
      " |             error is raised for multi-class classification.\n",
      " |  \n",
      " |      - When ``str``, specify the class targeted by the resampling. The\n",
      " |        number of samples in the different classes will be equalized.\n",
      " |        Possible choices are:\n",
      " |  \n",
      " |          ``'minority'``: resample only the minority class;\n",
      " |  \n",
      " |          ``'not minority'``: resample all classes but the minority class;\n",
      " |  \n",
      " |          ``'not majority'``: resample all classes but the majority class;\n",
      " |  \n",
      " |          ``'all'``: resample all classes;\n",
      " |  \n",
      " |          ``'auto'``: equivalent to ``'not majority'``.\n",
      " |  \n",
      " |      - When ``dict``, the keys correspond to the targeted classes. The\n",
      " |        values correspond to the desired number of samples for each targeted\n",
      " |        class.\n",
      " |  \n",
      " |      - When callable, function taking ``y`` and returns a ``dict``. The keys\n",
      " |        correspond to the targeted classes. The values correspond to the\n",
      " |        desired number of samples for each class.\n",
      " |  \n",
      " |  random_state : int, RandomState instance or None, optional (default=None)\n",
      " |      Control the randomization of the algorithm.\n",
      " |  \n",
      " |      - If int, ``random_state`` is the seed used by the random number\n",
      " |        generator;\n",
      " |      - If ``RandomState`` instance, random_state is the random number\n",
      " |        generator;\n",
      " |      - If ``None``, the random number generator is the ``RandomState``\n",
      " |        instance used by ``np.random``.\n",
      " |  \n",
      " |  k_neighbors : int or object, optional (default=5)\n",
      " |      If ``int``, number of nearest neighbours to used to construct synthetic\n",
      " |      samples.  If object, an estimator that inherits from\n",
      " |      :class:`sklearn.neighbors.base.KNeighborsMixin` that will be used to\n",
      " |      find the k_neighbors.\n",
      " |  \n",
      " |  n_jobs : int, optional (default=1)\n",
      " |      The number of threads to open if possible.\n",
      " |  \n",
      " |  Notes\n",
      " |  -----\n",
      " |  See the original paper [1]_ for more details.\n",
      " |  \n",
      " |  Supports mutli-class resampling. A one-vs.-rest scheme is used as\n",
      " |  originally proposed in [1]_.\n",
      " |  \n",
      " |  See\n",
      " |  :ref:`sphx_glr_auto_examples_over-sampling_plot_comparison_over_sampling.py`,\n",
      " |  and :ref:`sphx_glr_auto_examples_over-sampling_plot_smote.py`.\n",
      " |  \n",
      " |  See also\n",
      " |  --------\n",
      " |  SMOTE : Over-sample using SMOTE.\n",
      " |  \n",
      " |  SVMSMOTE : Over-sample using SVM-SMOTE variant.\n",
      " |  \n",
      " |  BorderlineSMOTE : Over-sample using Borderline-SMOTE variant.\n",
      " |  \n",
      " |  ADASYN : Over-sample using ADASYN.\n",
      " |  \n",
      " |  References\n",
      " |  ----------\n",
      " |  .. [1] N. V. Chawla, K. W. Bowyer, L. O.Hall, W. P. Kegelmeyer, \"SMOTE:\n",
      " |     synthetic minority over-sampling technique,\" Journal of artificial\n",
      " |     intelligence research, 321-357, 2002.\n",
      " |  \n",
      " |  Examples\n",
      " |  --------\n",
      " |  \n",
      " |  >>> from collections import Counter\n",
      " |  >>> from numpy.random import RandomState\n",
      " |  >>> from sklearn.datasets import make_classification\n",
      " |  >>> from imblearn.over_sampling import SMOTENC\n",
      " |  >>> X, y = make_classification(n_classes=2, class_sep=2,\n",
      " |  ... weights=[0.1, 0.9], n_informative=3, n_redundant=1, flip_y=0,\n",
      " |  ... n_features=20, n_clusters_per_class=1, n_samples=1000, random_state=10)\n",
      " |  >>> print('Original dataset shape (%s, %s)' % X.shape)\n",
      " |  Original dataset shape (1000, 20)\n",
      " |  >>> print('Original dataset samples per class {}'.format(Counter(y)))\n",
      " |  Original dataset samples per class Counter({1: 900, 0: 100})\n",
      " |  >>> # simulate the 2 last columns to be categorical features\n",
      " |  >>> X[:, -2:] = RandomState(10).randint(0, 4, size=(1000, 2))\n",
      " |  >>> sm = SMOTENC(random_state=42, categorical_features=[18, 19])\n",
      " |  >>> X_res, y_res = sm.fit_resample(X, y)\n",
      " |  >>> print('Resampled dataset samples per class {}'.format(Counter(y_res)))\n",
      " |  Resampled dataset samples per class Counter({0: 900, 1: 900})\n",
      " |  \n",
      " |  Method resolution order:\n",
      " |      SMOTENC\n",
      " |      SMOTE\n",
      " |      SVMSMOTE\n",
      " |      BorderlineSMOTE\n",
      " |      BaseSMOTE\n",
      " |      imblearn.over_sampling.base.BaseOverSampler\n",
      " |      imblearn.base.BaseSampler\n",
      " |      imblearn.base.SamplerMixin\n",
      " |      sklearn.base.BaseEstimator\n",
      " |      builtins.object\n",
      " |  \n",
      " |  Methods defined here:\n",
      " |  \n",
      " |  __init__(self, categorical_features, sampling_strategy='auto', random_state=None, k_neighbors=5, n_jobs=1)\n",
      " |      Initialize self.  See help(type(self)) for accurate signature.\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data and other attributes defined here:\n",
      " |  \n",
      " |  __abstractmethods__ = frozenset()\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors inherited from imblearn.base.BaseSampler:\n",
      " |  \n",
      " |  ratio_\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from imblearn.base.SamplerMixin:\n",
      " |  \n",
      " |  fit(self, X, y)\n",
      " |      Check inputs and statistics of the sampler.\n",
      " |      \n",
      " |      You should use ``fit_resample`` in all cases.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      X : {array-like, sparse matrix}, shape (n_samples, n_features)\n",
      " |          Data array.\n",
      " |      \n",
      " |      y : array-like, shape (n_samples,)\n",
      " |          Target array.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      self : object\n",
      " |          Return the instance itself.\n",
      " |  \n",
      " |  fit_resample(self, X, y)\n",
      " |      Resample the dataset.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      X : {array-like, sparse matrix}, shape (n_samples, n_features)\n",
      " |          Matrix containing the data which have to be sampled.\n",
      " |      \n",
      " |      y : array-like, shape (n_samples,)\n",
      " |          Corresponding label for each sample in X.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      X_resampled : {array-like, sparse matrix}, shape (n_samples_new, n_features)\n",
      " |          The array containing the resampled data.\n",
      " |      \n",
      " |      y_resampled : array-like, shape (n_samples_new,)\n",
      " |          The corresponding label of `X_resampled`.\n",
      " |  \n",
      " |  fit_sample = fit_resample(self, X, y)\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Methods inherited from sklearn.base.BaseEstimator:\n",
      " |  \n",
      " |  __getstate__(self)\n",
      " |  \n",
      " |  __repr__(self, N_CHAR_MAX=700)\n",
      " |      Return repr(self).\n",
      " |  \n",
      " |  __setstate__(self, state)\n",
      " |  \n",
      " |  get_params(self, deep=True)\n",
      " |      Get parameters for this estimator.\n",
      " |      \n",
      " |      Parameters\n",
      " |      ----------\n",
      " |      deep : boolean, optional\n",
      " |          If True, will return the parameters for this estimator and\n",
      " |          contained subobjects that are estimators.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      params : mapping of string to any\n",
      " |          Parameter names mapped to their values.\n",
      " |  \n",
      " |  set_params(self, **params)\n",
      " |      Set the parameters of this estimator.\n",
      " |      \n",
      " |      The method works on simple estimators as well as on nested objects\n",
      " |      (such as pipelines). The latter have parameters of the form\n",
      " |      ``<component>__<parameter>`` so that it's possible to update each\n",
      " |      component of a nested object.\n",
      " |      \n",
      " |      Returns\n",
      " |      -------\n",
      " |      self\n",
      " |  \n",
      " |  ----------------------------------------------------------------------\n",
      " |  Data descriptors inherited from sklearn.base.BaseEstimator:\n",
      " |  \n",
      " |  __dict__\n",
      " |      dictionary for instance variables (if defined)\n",
      " |  \n",
      " |  __weakref__\n",
      " |      list of weak references to the object (if defined)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(SMOTENC) # Synthetic Minority Over-sampling Technique for Nominal and Continuous"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_16 = pd.read_csv('../../Data/model_inputs/gdf_2016_X.csv')\n",
    "y_16 = pd.read_csv('../../Data/model_inputs/gdf_2016_y.csv')\n",
    "X_17 = pd.read_csv('../../Data/model_inputs/gdf_2017_X.csv')\n",
    "y_17 = pd.read_csv('../../Data/model_inputs/gdf_2017_y.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_trip</th>\n",
       "      <th>mode</th>\n",
       "      <th>duration</th>\n",
       "      <th>distance_m</th>\n",
       "      <th>weekday</th>\n",
       "      <th>precip</th>\n",
       "      <th>temp</th>\n",
       "      <th>morning</th>\n",
       "      <th>midday</th>\n",
       "      <th>afternoon</th>\n",
       "      <th>evening</th>\n",
       "      <th>midnight</th>\n",
       "      <th>startx</th>\n",
       "      <th>starty</th>\n",
       "      <th>endx</th>\n",
       "      <th>endy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>150744</td>\n",
       "      <td>2</td>\n",
       "      <td>862</td>\n",
       "      <td>9935.922336</td>\n",
       "      <td>1</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>16.910884</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>7.624322e+06</td>\n",
       "      <td>1.247673e+06</td>\n",
       "      <td>7.631864e+06</td>\n",
       "      <td>1.250415e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>35763</td>\n",
       "      <td>2</td>\n",
       "      <td>1873</td>\n",
       "      <td>6832.113937</td>\n",
       "      <td>1</td>\n",
       "      <td>9.972328e-07</td>\n",
       "      <td>18.007062</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>7.641919e+06</td>\n",
       "      <td>1.236661e+06</td>\n",
       "      <td>7.641941e+06</td>\n",
       "      <td>1.236732e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>323826</td>\n",
       "      <td>2</td>\n",
       "      <td>1165</td>\n",
       "      <td>12233.968564</td>\n",
       "      <td>1</td>\n",
       "      <td>9.972328e-07</td>\n",
       "      <td>18.007062</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>7.625699e+06</td>\n",
       "      <td>1.248427e+06</td>\n",
       "      <td>7.616671e+06</td>\n",
       "      <td>1.252917e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>375668</td>\n",
       "      <td>2</td>\n",
       "      <td>820</td>\n",
       "      <td>3612.405991</td>\n",
       "      <td>1</td>\n",
       "      <td>1.495849e-06</td>\n",
       "      <td>20.940647</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>7.630948e+06</td>\n",
       "      <td>1.253394e+06</td>\n",
       "      <td>7.630941e+06</td>\n",
       "      <td>1.253445e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>212877</td>\n",
       "      <td>2</td>\n",
       "      <td>1316</td>\n",
       "      <td>26428.385191</td>\n",
       "      <td>1</td>\n",
       "      <td>1.495849e-06</td>\n",
       "      <td>23.092603</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>7.631318e+06</td>\n",
       "      <td>1.252962e+06</td>\n",
       "      <td>7.646416e+06</td>\n",
       "      <td>1.264378e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id_trip  mode  duration    distance_m  weekday        precip       temp  \\\n",
       "0   150744     2       862   9935.922336        1  0.000000e+00  16.910884   \n",
       "1    35763     2      1873   6832.113937        1  9.972328e-07  18.007062   \n",
       "2   323826     2      1165  12233.968564        1  9.972328e-07  18.007062   \n",
       "3   375668     2       820   3612.405991        1  1.495849e-06  20.940647   \n",
       "4   212877     2      1316  26428.385191        1  1.495849e-06  23.092603   \n",
       "\n",
       "   morning  midday  afternoon  evening  midnight        startx        starty  \\\n",
       "0     True   False      False    False     False  7.624322e+06  1.247673e+06   \n",
       "1    False    True      False    False     False  7.641919e+06  1.236661e+06   \n",
       "2    False    True      False    False     False  7.625699e+06  1.248427e+06   \n",
       "3    False    True      False    False     False  7.630948e+06  1.253394e+06   \n",
       "4    False   False       True    False     False  7.631318e+06  1.252962e+06   \n",
       "\n",
       "           endx          endy  \n",
       "0  7.631864e+06  1.250415e+06  \n",
       "1  7.641941e+06  1.236732e+06  \n",
       "2  7.616671e+06  1.252917e+06  \n",
       "3  7.630941e+06  1.253445e+06  \n",
       "4  7.646416e+06  1.264378e+06  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_17.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique number of ids = 22948\n"
     ]
    }
   ],
   "source": [
    "print('unique number of ids =', X_17.id_trip.nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## for oversampling\n",
    "# ros = RandomOverSampler(random_state=0)\n",
    "# ros = SMOTE(random_state=0)\n",
    "categorical_features = [True,True,False,False,True,False,False,True,True,True,True,True,False,False,False,False]\n",
    "ros = SMOTENC(categorical_features, random_state=0)\n",
    "\n",
    "## NOTE: x and y of start and end of trip will be non-categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['id_trip', 'mode', 'weekday', 'morning', 'midday', 'afternoon',\n",
       "       'evening', 'midnight'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_17.columns[categorical_features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Applications/anaconda/envs/st-ds/lib/python3.7/site-packages/sklearn/utils/validation.py:724: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n",
      "/Applications/anaconda/envs/st-ds/lib/python3.7/site-packages/sklearn/utils/validation.py:724: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  y = column_or_1d(y, warn=True)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2min 19s, sys: 4.09 s, total: 2min 23s\n",
      "Wall time: 2min 30s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "oversample_X_16, oversample_y_16 = ros.fit_resample(X_16, y_16)\n",
    "oversample_X_17, oversample_y_17 = ros.fit_resample(X_17, y_17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Before: num of classes\n",
      " 0    7430\n",
      "1    5790\n",
      "2    5682\n",
      "3    2473\n",
      "4    2262\n",
      "5    2168\n",
      "Name: purpose, dtype: int64\n",
      "Before: num of classes\n",
      " 0    8049\n",
      "2    7723\n",
      "1    2714\n",
      "4    2175\n",
      "3    1379\n",
      "5     908\n",
      "Name: purpose, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print('Before: num of classes\\n', y_16['purpose'].value_counts())\n",
    "print('Before: num of classes\\n', y_17['purpose'].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After: num of classes\n",
      " 5    7430\n",
      "4    7430\n",
      "3    7430\n",
      "2    7430\n",
      "1    7430\n",
      "0    7430\n",
      "Name: 0, dtype: int64\n",
      "After: num of classes\n",
      " 5    8049\n",
      "4    8049\n",
      "3    8049\n",
      "2    8049\n",
      "1    8049\n",
      "0    8049\n",
      "Name: 0, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print('After: num of classes\\n', pd.DataFrame(oversample_y_16)[0].value_counts())\n",
    "print('After: num of classes\\n', pd.DataFrame(oversample_y_17)[0].value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "over_X_16 = pd.DataFrame(oversample_X_16, columns=X_16.columns)\n",
    "over_y_16 = pd.DataFrame(oversample_y_16, columns=['purpose'])\n",
    "over_X_17 = pd.DataFrame(oversample_X_17, columns=X_17.columns)\n",
    "over_y_17 = pd.DataFrame(oversample_y_17, columns=['purpose'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "460889    112\n",
       "473372     87\n",
       "463145     85\n",
       "426797     79\n",
       "454196     79\n",
       "         ... \n",
       "250515      1\n",
       "268087      1\n",
       "112495      1\n",
       "426967      1\n",
       "294912      1\n",
       "Name: id_trip, Length: 22948, dtype: int64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "over_X_17['id_trip'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id_trip</th>\n",
       "      <th>mode</th>\n",
       "      <th>duration</th>\n",
       "      <th>distance_m</th>\n",
       "      <th>weekday</th>\n",
       "      <th>precip</th>\n",
       "      <th>temp</th>\n",
       "      <th>morning</th>\n",
       "      <th>midday</th>\n",
       "      <th>afternoon</th>\n",
       "      <th>evening</th>\n",
       "      <th>midnight</th>\n",
       "      <th>startx</th>\n",
       "      <th>starty</th>\n",
       "      <th>endx</th>\n",
       "      <th>endy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13263</th>\n",
       "      <td>460889</td>\n",
       "      <td>2</td>\n",
       "      <td>1071</td>\n",
       "      <td>4147.4</td>\n",
       "      <td>1</td>\n",
       "      <td>1.04709e-05</td>\n",
       "      <td>13.8305</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>7.61876e+06</td>\n",
       "      <td>1.24685e+06</td>\n",
       "      <td>7.61773e+06</td>\n",
       "      <td>1.24742e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41162</th>\n",
       "      <td>460889</td>\n",
       "      <td>2</td>\n",
       "      <td>1182.6</td>\n",
       "      <td>3321.53</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>13.0877</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>7.6192e+06</td>\n",
       "      <td>1.24877e+06</td>\n",
       "      <td>7.62059e+06</td>\n",
       "      <td>1.25033e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41192</th>\n",
       "      <td>460889</td>\n",
       "      <td>2</td>\n",
       "      <td>1459.64</td>\n",
       "      <td>528.097</td>\n",
       "      <td>1</td>\n",
       "      <td>1.1554e-06</td>\n",
       "      <td>16.9751</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>7.619e+06</td>\n",
       "      <td>1.24373e+06</td>\n",
       "      <td>7.61863e+06</td>\n",
       "      <td>1.2438e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41198</th>\n",
       "      <td>460889</td>\n",
       "      <td>0</td>\n",
       "      <td>1400.23</td>\n",
       "      <td>440.245</td>\n",
       "      <td>1</td>\n",
       "      <td>1.49585e-06</td>\n",
       "      <td>17.4179</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>7.619e+06</td>\n",
       "      <td>1.24373e+06</td>\n",
       "      <td>7.61863e+06</td>\n",
       "      <td>1.2438e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41214</th>\n",
       "      <td>460889</td>\n",
       "      <td>2</td>\n",
       "      <td>708.009</td>\n",
       "      <td>379.4</td>\n",
       "      <td>1</td>\n",
       "      <td>1.49585e-06</td>\n",
       "      <td>20.8406</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>7.61894e+06</td>\n",
       "      <td>1.24376e+06</td>\n",
       "      <td>7.61863e+06</td>\n",
       "      <td>1.2438e+06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id_trip mode duration distance_m weekday       precip     temp morning  \\\n",
       "13263  460889    2     1071     4147.4       1  1.04709e-05  13.8305   False   \n",
       "41162  460889    2   1182.6    3321.53       1            0  13.0877   False   \n",
       "41192  460889    2  1459.64    528.097       1   1.1554e-06  16.9751   False   \n",
       "41198  460889    0  1400.23    440.245       1  1.49585e-06  17.4179   False   \n",
       "41214  460889    2  708.009      379.4       1  1.49585e-06  20.8406   False   \n",
       "\n",
       "      midday afternoon evening midnight       startx       starty  \\\n",
       "13263  False      True   False    False  7.61876e+06  1.24685e+06   \n",
       "41162  False      True   False    False   7.6192e+06  1.24877e+06   \n",
       "41192  False      True   False    False    7.619e+06  1.24373e+06   \n",
       "41198  False      True   False    False    7.619e+06  1.24373e+06   \n",
       "41214  False      True   False    False  7.61894e+06  1.24376e+06   \n",
       "\n",
       "              endx         endy  \n",
       "13263  7.61773e+06  1.24742e+06  \n",
       "41162  7.62059e+06  1.25033e+06  \n",
       "41192  7.61863e+06   1.2438e+06  \n",
       "41198  7.61863e+06   1.2438e+06  \n",
       "41214  7.61863e+06   1.2438e+06  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "over_X_17.loc[over_X_17['id_trip'] == 460889].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "unique number of ids = 22948\n"
     ]
    }
   ],
   "source": [
    "print('unique number of ids =',over_X_17.id_trip.nunique())\n",
    "## same amount"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## save data if needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# over_X_16.to_csv('../../Data/model_inputs/oversampled_X_2016.csv',index=False)\n",
    "# over_y_16.to_csv('../../Data/model_inputs/oversampled_y_2016.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# over_X_17.to_csv('../../Data/model_inputs/oversampled_X_2017.csv',index=False)\n",
    "# over_y_17.to_csv('../../Data/model_inputs/oversampled_y_2017.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
